{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "构建基类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    \"\"\"\n",
    "    Each node in neural networks will have these attributes and methods\n",
    "    \"\"\"\n",
    "    def __init__(self, inputs=[]):\n",
    "        self.inputs = inputs\n",
    "        self.value = None\n",
    "        self.outputs = []\n",
    "        self.gradients = {}\n",
    "        \n",
    "        for node in self.inputs:\n",
    "            node.outputs.append(self)\n",
    "            \n",
    "    def forward(self):\n",
    "        \"\"\"\n",
    "        Forward propogation\n",
    "        \n",
    "        Compute the output value based on input nodes and store the value\n",
    "        \"\"\"\n",
    "        raise NotImplemented\n",
    "        \n",
    "    def backward(self):\n",
    "        \"\"\"\n",
    "        Back propogation\n",
    "        \n",
    "        Compute the gradient of each input node and store the value\n",
    "        \"\"\"\n",
    "        raise NotImplemented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Input(Node):\n",
    "    def __init__(self, name=''):\n",
    "        Node.__init__(self, inputs=[])\n",
    "        self.name = name\n",
    "        \n",
    "    def forward(self, value=None):\n",
    "        if value is not None:\n",
    "            self.value = value\n",
    "            \n",
    "    def backward(self):\n",
    "        self.gradients = {}\n",
    "        \n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self] = grad_cost\n",
    "            \n",
    "    def __repr__(self):\n",
    "        return 'Input Node: {}'.format(self.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Node):\n",
    "    def __init__(self, nodes, weights, bias):\n",
    "        self.w_node = weights\n",
    "        self.x_node = nodes\n",
    "        self.b_node = bias\n",
    "        Node.__init__(self, inputs=[nodes, weights, bias])\n",
    "        \n",
    "    def forward(self):\n",
    "        '''\n",
    "        Compute wx + b by using numpy\n",
    "        '''\n",
    "        self.value = np.dot(self.x_node.value, self.w_node.value) + self.b_node.value\n",
    "        \n",
    "    def backward(self):\n",
    "        for node in self.outputs:\n",
    "            grad_cost = node.gradients[self]\n",
    "            \n",
    "            self.gradients[self.w_node] = np.dot(self.x_node.value.T, grad_cost)\n",
    "            self.gradients[self.b_node] = np.sum(grad_cost * 1, axis=0, keepdims=False)\n",
    "            self.gradients[self.x_node] = np.dot(grad_cost, self.w_node.value.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid(Node):\n",
    "    def __init__(self, node):\n",
    "        Node.__init__(self, [node])\n",
    "        self.x_node = node\n",
    "        \n",
    "    def _sigmoid(self, x):\n",
    "        return 1. / (1 + np.exp(-1 * x))\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = self._sigmoid(self.x_node.value)\n",
    "        \n",
    "    def backward(self):\n",
    "        y = self.value\n",
    "        \n",
    "        self.partial = y * (1 - y)\n",
    "        \n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.x_node] = grad_cost * self.partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MSE(Node):\n",
    "    def __init__(self, y_true, y_hat):\n",
    "        self.y_true_node = y_true\n",
    "        self.y_hat_node = y_hat\n",
    "        Node.__init__(self, inputs=[y_true, y_hat])\n",
    "        \n",
    "    def forward(self):\n",
    "        y_true_flatten = self.y_true_node.value.reshape(-1, 1)\n",
    "        y_hat_flatten = self.y_hat_node.value.reshape(-1, 1)\n",
    "        \n",
    "        self.diff = y_true_flatten - y_hat_flatten\n",
    "        \n",
    "        self.value = np.mean(self.diff ** 2)\n",
    "        \n",
    "    def backward(self):\n",
    "        n = self.y_hat_node.value.shape[0]\n",
    "        \n",
    "        self.gradients[self.y_true_node] = (2 / n) * self.diff\n",
    "        self.gradients[self.y_hat_node] = (-2 / n) * self.diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_one_batch(topological_sorted_graph):\n",
    "    # graph是经过拓补排序后的一个list\n",
    "    \n",
    "    for node in topological_sorted_graph:\n",
    "        node.forward()\n",
    "        \n",
    "    for node in topological_sorted_graph[::-1]:\n",
    "        node.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topological_sort(data_with_value):\n",
    "    feed_dict = data_with_value\n",
    "    input_nodes = [n for n in feed_dict.keys()]\n",
    "    \n",
    "    G = {}\n",
    "    nodes = [n for n in input_nodes]\n",
    "    while len(nodes) > 0:\n",
    "        n = nodes.pop(0)\n",
    "        if n not in G:\n",
    "            G[n] = {'in':set(), 'out':set()}\n",
    "        for m in n.outputs:\n",
    "            if m not in G:\n",
    "                G[m] = {'in':set(), 'out':set()}\n",
    "            G[n]['out'].add(m)\n",
    "            G[m]['in'].add(n)\n",
    "            nodes.append(m)\n",
    "            \n",
    "    L = []\n",
    "    S = set(input_nodes)\n",
    "    while len(S) > 0:\n",
    "        n = S.pop()\n",
    "        \n",
    "        if isinstance(n, Input):\n",
    "            n.value = feed_dict[n]\n",
    "            \n",
    "        L.append(n)\n",
    "        for m in n.outputs:\n",
    "            G[n]['out'].remove(m)\n",
    "            G[m]['in'].remove(n)\n",
    "            if len(G[m]['in']) == 0:\n",
    "                S.add(m)\n",
    "    return L"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sgd_update(trainable_nodes, learning_rate=1e-2):\n",
    "    for t in trainable_nodes:\n",
    "        t.value += -1 * learning_rate * t.gradients[t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_boston()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_ = data['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.3200e-03, 1.8000e+01, 2.3100e+00, ..., 1.5300e+01, 3.9690e+02,\n",
       "        4.9800e+00],\n",
       "       [2.7310e-02, 0.0000e+00, 7.0700e+00, ..., 1.7800e+01, 3.9690e+02,\n",
       "        9.1400e+00],\n",
       "       [2.7290e-02, 0.0000e+00, 7.0700e+00, ..., 1.7800e+01, 3.9283e+02,\n",
       "        4.0300e+00],\n",
       "       ...,\n",
       "       [6.0760e-02, 0.0000e+00, 1.1930e+01, ..., 2.1000e+01, 3.9690e+02,\n",
       "        5.6400e+00],\n",
       "       [1.0959e-01, 0.0000e+00, 1.1930e+01, ..., 2.1000e+01, 3.9345e+02,\n",
       "        6.4800e+00],\n",
       "       [4.7410e-02, 0.0000e+00, 1.1930e+01, ..., 2.1000e+01, 3.9690e+02,\n",
       "        7.8800e+00]])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_ = data['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([24. , 21.6, 34.7, 33.4, 36.2, 28.7, 22.9, 27.1, 16.5, 18.9, 15. ,\n",
       "       18.9, 21.7, 20.4, 18.2, 19.9, 23.1, 17.5, 20.2, 18.2, 13.6, 19.6,\n",
       "       15.2, 14.5, 15.6, 13.9, 16.6, 14.8, 18.4, 21. , 12.7, 14.5, 13.2,\n",
       "       13.1, 13.5, 18.9, 20. , 21. , 24.7, 30.8, 34.9, 26.6, 25.3, 24.7,\n",
       "       21.2, 19.3, 20. , 16.6, 14.4, 19.4, 19.7, 20.5, 25. , 23.4, 18.9,\n",
       "       35.4, 24.7, 31.6, 23.3, 19.6, 18.7, 16. , 22.2, 25. , 33. , 23.5,\n",
       "       19.4, 22. , 17.4, 20.9, 24.2, 21.7, 22.8, 23.4, 24.1, 21.4, 20. ,\n",
       "       20.8, 21.2, 20.3, 28. , 23.9, 24.8, 22.9, 23.9, 26.6, 22.5, 22.2,\n",
       "       23.6, 28.7, 22.6, 22. , 22.9, 25. , 20.6, 28.4, 21.4, 38.7, 43.8,\n",
       "       33.2, 27.5, 26.5, 18.6, 19.3, 20.1, 19.5, 19.5, 20.4, 19.8, 19.4,\n",
       "       21.7, 22.8, 18.8, 18.7, 18.5, 18.3, 21.2, 19.2, 20.4, 19.3, 22. ,\n",
       "       20.3, 20.5, 17.3, 18.8, 21.4, 15.7, 16.2, 18. , 14.3, 19.2, 19.6,\n",
       "       23. , 18.4, 15.6, 18.1, 17.4, 17.1, 13.3, 17.8, 14. , 14.4, 13.4,\n",
       "       15.6, 11.8, 13.8, 15.6, 14.6, 17.8, 15.4, 21.5, 19.6, 15.3, 19.4,\n",
       "       17. , 15.6, 13.1, 41.3, 24.3, 23.3, 27. , 50. , 50. , 50. , 22.7,\n",
       "       25. , 50. , 23.8, 23.8, 22.3, 17.4, 19.1, 23.1, 23.6, 22.6, 29.4,\n",
       "       23.2, 24.6, 29.9, 37.2, 39.8, 36.2, 37.9, 32.5, 26.4, 29.6, 50. ,\n",
       "       32. , 29.8, 34.9, 37. , 30.5, 36.4, 31.1, 29.1, 50. , 33.3, 30.3,\n",
       "       34.6, 34.9, 32.9, 24.1, 42.3, 48.5, 50. , 22.6, 24.4, 22.5, 24.4,\n",
       "       20. , 21.7, 19.3, 22.4, 28.1, 23.7, 25. , 23.3, 28.7, 21.5, 23. ,\n",
       "       26.7, 21.7, 27.5, 30.1, 44.8, 50. , 37.6, 31.6, 46.7, 31.5, 24.3,\n",
       "       31.7, 41.7, 48.3, 29. , 24. , 25.1, 31.5, 23.7, 23.3, 22. , 20.1,\n",
       "       22.2, 23.7, 17.6, 18.5, 24.3, 20.5, 24.5, 26.2, 24.4, 24.8, 29.6,\n",
       "       42.8, 21.9, 20.9, 44. , 50. , 36. , 30.1, 33.8, 43.1, 48.8, 31. ,\n",
       "       36.5, 22.8, 30.7, 50. , 43.5, 20.7, 21.1, 25.2, 24.4, 35.2, 32.4,\n",
       "       32. , 33.2, 33.1, 29.1, 35.1, 45.4, 35.4, 46. , 50. , 32.2, 22. ,\n",
       "       20.1, 23.2, 22.3, 24.8, 28.5, 37.3, 27.9, 23.9, 21.7, 28.6, 27.1,\n",
       "       20.3, 22.5, 29. , 24.8, 22. , 26.4, 33.1, 36.1, 28.4, 33.4, 28.2,\n",
       "       22.8, 20.3, 16.1, 22.1, 19.4, 21.6, 23.8, 16.2, 17.8, 19.8, 23.1,\n",
       "       21. , 23.8, 23.1, 20.4, 18.5, 25. , 24.6, 23. , 22.2, 19.3, 22.6,\n",
       "       19.8, 17.1, 19.4, 22.2, 20.7, 21.1, 19.5, 18.5, 20.6, 19. , 18.7,\n",
       "       32.7, 16.5, 23.9, 31.2, 17.5, 17.2, 23.1, 24.5, 26.6, 22.9, 24.1,\n",
       "       18.6, 30.1, 18.2, 20.6, 17.8, 21.7, 22.7, 22.6, 25. , 19.9, 20.8,\n",
       "       16.8, 21.9, 27.5, 21.9, 23.1, 50. , 50. , 50. , 50. , 50. , 13.8,\n",
       "       13.8, 15. , 13.9, 13.3, 13.1, 10.2, 10.4, 10.9, 11.3, 12.3,  8.8,\n",
       "        7.2, 10.5,  7.4, 10.2, 11.5, 15.1, 23.2,  9.7, 13.8, 12.7, 13.1,\n",
       "       12.5,  8.5,  5. ,  6.3,  5.6,  7.2, 12.1,  8.3,  8.5,  5. , 11.9,\n",
       "       27.9, 17.2, 27.5, 15. , 17.2, 17.9, 16.3,  7. ,  7.2,  7.5, 10.4,\n",
       "        8.8,  8.4, 16.7, 14.2, 20.8, 13.4, 11.7,  8.3, 10.2, 10.9, 11. ,\n",
       "        9.5, 14.5, 14.1, 16.1, 14.3, 11.7, 13.4,  9.6,  8.7,  8.4, 12.8,\n",
       "       10.5, 17.1, 18.4, 15.4, 10.8, 11.8, 14.9, 12.6, 14.1, 13. , 13.4,\n",
       "       15.2, 16.1, 17.8, 14.9, 14.1, 12.7, 13.5, 14.9, 20. , 16.4, 17.7,\n",
       "       19.5, 20.2, 21.4, 19.9, 19. , 19.1, 19.1, 20.1, 19.9, 19.6, 23.2,\n",
       "       29.8, 13.8, 13.3, 16.7, 12. , 14.6, 21.4, 23. , 23.7, 25. , 21.8,\n",
       "       20.6, 21.2, 19.1, 20.6, 15.2,  7. ,  8.1, 13.6, 20.1, 21.8, 24.5,\n",
       "       23.1, 19.7, 18.3, 21.2, 17.5, 16.8, 22.4, 20.6, 23.9, 22. , 11.9])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_ = (X_ - np.mean(X_, axis=0)) / np.std(X_, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_features = X_.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1_, b1_ = np.random.randn(n_features, n_hidden), np.zeros(n_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "W2_, b2_ = np.random.randn(n_hidden, 1), np.zeros(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a graph connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build nodes in this graph\n",
    "# tensorflow >> placeholder\n",
    "X, y = Input(name='X'), Input(name='y')\n",
    "W1, b1 = Input(name='W1'), Input(name='b1')\n",
    "W2, b2 = Input(name='W2'), Input(name='b2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build connection ralationship\n",
    "liner_output = Linear(X, W1, b1)\n",
    "sigmoid_output = Sigmoid(liner_output)\n",
    "yhat = Linear(sigmoid_output, W2, b2)\n",
    "loss = MSE(y, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_with_value = {\n",
    "    X: X_,\n",
    "    y: y_,\n",
    "    W1: W1_,\n",
    "    b1: b1_,\n",
    "    W2: W2_,\n",
    "    b2: b2_\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "graph = topological_sort(input_node_with_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Input Node: X,\n",
       " Input Node: b1,\n",
       " Input Node: b2,\n",
       " Input Node: y,\n",
       " Input Node: W1,\n",
       " Input Node: W2,\n",
       " <__main__.Linear at 0x23533b214a8>,\n",
       " <__main__.Sigmoid at 0x23533b21320>,\n",
       " <__main__.Linear at 0x23533b215c0>,\n",
       " <__main__.MSE at 0x23533b21898>]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, loss: 576.7456936359741\n",
      "Epoch: 101, loss: 28.359273297249576\n",
      "Epoch: 201, loss: 23.60741907660276\n",
      "Epoch: 301, loss: 17.417465478619707\n",
      "Epoch: 401, loss: 18.035661837386705\n",
      "Epoch: 501, loss: 17.361233080085036\n",
      "Epoch: 601, loss: 10.634362384303435\n",
      "Epoch: 701, loss: 11.660346427132103\n",
      "Epoch: 801, loss: 13.638418042352729\n",
      "Epoch: 901, loss: 10.916630182451362\n",
      "Epoch: 1001, loss: 9.526471441993563\n",
      "Epoch: 1101, loss: 8.95878484671576\n",
      "Epoch: 1201, loss: 9.527506842716432\n",
      "Epoch: 1301, loss: 8.420601226355133\n",
      "Epoch: 1401, loss: 9.80021526553408\n",
      "Epoch: 1501, loss: 8.606060961968625\n",
      "Epoch: 1601, loss: 8.237449975251403\n",
      "Epoch: 1701, loss: 8.098925583130365\n",
      "Epoch: 1801, loss: 7.0478398625570104\n",
      "Epoch: 1901, loss: 8.927579775205306\n",
      "Epoch: 2001, loss: 6.990845228369402\n",
      "Epoch: 2101, loss: 8.262029216051529\n",
      "Epoch: 2201, loss: 7.067049902949029\n",
      "Epoch: 2301, loss: 8.827980507262202\n",
      "Epoch: 2401, loss: 6.86032090812241\n",
      "Epoch: 2501, loss: 7.942791362983644\n",
      "Epoch: 2601, loss: 8.029056774618477\n",
      "Epoch: 2701, loss: 7.2504905012001135\n",
      "Epoch: 2801, loss: 5.722502349844656\n",
      "Epoch: 2901, loss: 7.191983795674304\n",
      "Epoch: 3001, loss: 8.01968661416638\n",
      "Epoch: 3101, loss: 8.638048742147506\n",
      "Epoch: 3201, loss: 5.749562801163097\n",
      "Epoch: 3301, loss: 5.578784887269363\n",
      "Epoch: 3401, loss: 6.419509358622642\n",
      "Epoch: 3501, loss: 6.132701955975783\n",
      "Epoch: 3601, loss: 7.389439367314507\n",
      "Epoch: 3701, loss: 6.435945419537879\n",
      "Epoch: 3801, loss: 6.430084939772375\n",
      "Epoch: 3901, loss: 5.982075741668983\n",
      "Epoch: 4001, loss: 6.2977534338935595\n",
      "Epoch: 4101, loss: 7.331176290198071\n",
      "Epoch: 4201, loss: 6.883038845125474\n",
      "Epoch: 4301, loss: 5.870110599438784\n",
      "Epoch: 4401, loss: 6.817016502325008\n",
      "Epoch: 4501, loss: 6.5577028829718484\n",
      "Epoch: 4601, loss: 6.810154207669092\n",
      "Epoch: 4701, loss: 5.943073348611741\n",
      "Epoch: 4801, loss: 5.497921137555907\n",
      "Epoch: 4901, loss: 6.571454312968308\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "\n",
    "losses = []\n",
    "epochs = 5000\n",
    "batch_size = 64\n",
    "\n",
    "steps_per_epoch = X_.shape[0] // batch_size\n",
    "\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "    \n",
    "    for batch in range(steps_per_epoch):\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "        \n",
    "        training_one_batch(graph)\n",
    "        learning_rate = 1e-3\n",
    "        \n",
    "        sgd_update(trainable_nodes=[W1, W2, b1, b2], learning_rate=learning_rate)\n",
    "        \n",
    "        loss += graph[-1].value\n",
    "    if i % 100 == 0:\n",
    "        print('Epoch: {}, loss: {}'.format(i + 1, loss / steps_per_epoch))\n",
    "        losses.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x235320909b0>]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAfCElEQVR4nO3de4xc5Znn8e+vLl3l+wXaBGxn7CTODCRanIyXsMtoNgMZMExmzKyCRDQ7sSIkz2qJNpGyOwP5h1wGbSLthNlIk0hM8MaJMiEol8XLeofxEqJssgrQ3DEkQ4cQcGzsBt8vXV2XZ/84b3VXt7vb3e3udqjz+0ilc85b51S9b7v8nKfe89Z5FRGYmVk+FM53BczMbP446JuZ5YiDvplZjjjom5nliIO+mVmOlM53BSZz4YUXxrp16853NczM3lQef/zx1yOid7znfqOD/rp16+jr6zvf1TAze1OR9KuJnnP3jplZjjjom5nliIO+mVmOOOibmeWIg76ZWY446JuZ5YiDvplZjkw56EsqSnpS0gNpe72kRyS9KOnbknpSeSVt96fn13W8xu2p/OeSrpvtxrSdqDX44u5/5slXDs/VW5iZvSlNJ9P/OPBCx/YXgLsiYgNwGLglld8CHI6IdwB3pf2QdBlwM/AuYDPwZUnFc6v++OqNFl966EWefvXIXLy8mdmb1pSCvqQ1wB8BX03bAq4GvpN22QHcmNa3pG3S89ek/bcA90ZELSJ+CfQDV8xGI8aqlLNmDTZac/HyZmZvWlPN9P8W+EugHUUvAI5ERCNt7wVWp/XVwKsA6fmjaf/h8nGOGSZpm6Q+SX0DAwPTaMqISin7AlGrO+ibmXU6a9CX9EHgYEQ83lk8zq5xlucmO2akIOLuiNgUEZt6e8e9X9BZFQuiXBS1RnNGx5uZdaup3HDtKuBPJN0AVIGlZJn/ckmllM2vAfal/fcCa4G9kkrAMuBQR3lb5zGzrlIqMuhM38xslLNm+hFxe0SsiYh1ZBdifxARfwY8DHwo7bYVuD+t70zbpOd/ENns6zuBm9PonvXABuDRWWvJGJVSwZm+mdkY53Jr5b8C7pX018CTwD2p/B7gG5L6yTL8mwEiYo+k+4DngQZwa0TMWVSulp3pm5mNNa2gHxE/BH6Y1l9inNE3ETEI3DTB8XcCd063kjPhTN/M7Exd+4vcSrlIzUM2zcxG6d6gXyowWHemb2bWqauDvjN9M7PRujboV929Y2Z2hq4N+pVSgZq7d8zMRuneoO9M38zsDF0b9Ku+kGtmdoauDfqVsi/kmpmN1bVBv1oquk/fzGyMrg36lXLB99M3Mxuje4N+qUizFTSaDvxmZm1dG/SrafYs9+ubmY3o2qDfnj3LI3jMzEZ0cdB3pm9mNlbXBv1q2Zm+mdlYXRv0nembmZ1pKhOjVyU9KulpSXskfSaVf03SLyU9lR4bU7kkfUlSv6RnJL2347W2SnoxPbZO9J6zoZ3pO+ibmY2YysxZNeDqiDghqQz8WNL/Ts/954j4zpj9ryeb/3YD8D7gK8D7JK0E7gA2AQE8LmlnRByejYaM1c703b1jZjZiKhOjR0ScSJvl9IhJDtkCfD0d91NguaSLgeuA3RFxKAX63cDmc6v+xCoesmlmdoYp9elLKkp6CjhIFrgfSU/dmbpw7pJUSWWrgVc7Dt+byiYqH/te2yT1SeobGBiYZnNGtIds+lYMZmYjphT0I6IZERuBNcAVkt4N3A78DvAvgZXAX6XdNd5LTFI+9r3ujohNEbGpt7d3KtUbV/vHWb4Vg5nZiGmN3omII8APgc0RsT914dSA/w5ckXbbC6ztOGwNsG+S8jnhTN/M7ExTGb3TK2l5Wl8AfAD4WeqnR5KAG4Hn0iE7gY+kUTxXAkcjYj/wIHCtpBWSVgDXprI5UXGmb2Z2hqmM3rkY2CGpSHaSuC8iHpD0A0m9ZN02TwH/Pu2/C7gB6AdOAR8FiIhDkj4HPJb2+2xEHJq9pozmTN/M7ExnDfoR8QzwnnHKr55g/wBuneC57cD2adZxRnzDNTOzM3XtL3J7iinoO9M3MxvWtUFfEpWSp0w0M+vUtUEfslsxOOibmY3o6qBfKRV8GwYzsw7dHfTL7t4xM+vU1UG/Wio60zcz69DVQd+ZvpnZaF0d9KulIrWGM30zs7auDvqVcoHBujN9M7O27g76zvTNzEbp6qBfLReoOdM3MxvW1UG/Uioy6EzfzGxYVwd9Z/pmZqN1ddCveJy+mdkoXR70PU7fzKzTVGbOqkp6VNLTkvZI+kwqXy/pEUkvSvq2pJ5UXknb/en5dR2vdXsq/7mk6+aqUW2VdMO17Bb/ZmY2lUy/BlwdEZcDG4HNaRrELwB3RcQG4DBwS9r/FuBwRLwDuCvth6TLgJuBdwGbgS+n2bjmTKXkiVTMzDqdNeinyc9PpM1yegRwNfCdVL6DbJ5cgC1pm/T8NWke3S3AvRFRi4hfkk2n2J5MfU446JuZjTalPn1JRUlPAQeB3cAvgCMR0Ui77AVWp/XVwKsA6fmjwAWd5eMcMyeq5TRProdtmpkBUwz6EdGMiI3AGrLs/NLxdktLTfDcROWjSNomqU9S38DAwFSqN6HhTN/DNs3MgGmO3omII8APgSuB5ZLaE6uvAfal9b3AWoD0/DLgUGf5OMd0vsfdEbEpIjb19vZOp3pncKZvZjbaVEbv9EpantYXAB8AXgAeBj6UdtsK3J/Wd6Zt0vM/iGz4zE7g5jS6Zz2wAXh0thoynnam75uumZllSmffhYuBHWmkTQG4LyIekPQ8cK+kvwaeBO5J+98DfENSP1mGfzNAROyRdB/wPNAAbo2IOU3BK870zcxGOWvQj4hngPeMU/4S44y+iYhB4KYJXutO4M7pV3Nmqu7TNzMbpbt/kZsyfd90zcws091B35m+mdkoXR30R0bvOOibmUGXB/2R0Tvu3jEzgy4P+s70zcxG6+qg70zfzGy0XAR9Z/pmZpmuDvqlYoFSQf5xlplZ0tVBH7Js37dhMDPLdH/QLxed6ZuZJV0f9KvO9M3MhnV90G/Pk2tmZnkI+qUCNQ/ZNDMD8hD0y0UGnembmQF5CPrO9M3MhnV90K+6T9/MbNhUpktcK+lhSS9I2iPp46n805J+Lemp9Lih45jbJfVL+rmk6zrKN6eyfkm3zU2TRsvG6TvTNzODqU2X2AA+GRFPSFoCPC5pd3ruroj4r507S7qMbIrEdwGXAP9H0jvT038H/CHZJOmPSdoZEc/PRkMmUikVGHKmb2YGTG26xP3A/rR+XNILwOpJDtkC3BsRNeCXaa7c9rSK/WmaRSTdm/ad06BfLRed6ZuZJdPq05e0jmy+3EdS0cckPSNpu6QVqWw18GrHYXtT2UTlY99jm6Q+SX0DAwPTqd64KqWC+/TNzJIpB31Ji4HvAp+IiGPAV4C3AxvJvgn8TXvXcQ6PScpHF0TcHRGbImJTb2/vVKs3IV/INTMbMZU+fSSVyQL+NyPiewARcaDj+b8HHkibe4G1HYevAfal9YnK54wv5JqZjZjK6B0B9wAvRMQXO8ov7tjtT4Hn0vpO4GZJFUnrgQ3Ao8BjwAZJ6yX1kF3s3Tk7zZhYpVSk0QoaTWf7ZmZTyfSvAv4ceFbSU6nsU8CHJW0k66J5GfgLgIjYI+k+sgu0DeDWiGgCSPoY8CBQBLZHxJ5ZbMu4quXsvDbUbFEqdv3PEszMJjWV0Ts/Zvz++F2THHMncOc45bsmO24ujEyZ2GJhz3y+s5nZb56uT30rw5Oju1/fzKzrg367e8f31Dczy0HQr5Sc6ZuZtXV90G9n+jVn+mZm3R/025m+x+qbmeUi6KdM37/KNTPr/qBfHR6946BvZtb1QX9knL67d8zMchD0nembmbV1fdAfGafvTN/MrOuDvjN9M7MR3R/02+P0/eMsM7McBP2Sb8NgZtbW9UFfEj2lgjN9MzNyEPQBqqWCb8NgZkZOgn6lXHSmb2bG1KZLXCvpYUkvSNoj6eOpfKWk3ZJeTMsVqVySviSpX9Izkt7b8Vpb0/4vSto6d80arVp2pm9mBlPL9BvAJyPiUuBK4FZJlwG3AQ9FxAbgobQNcD3ZvLgbgG3AVyA7SQB3AO8DrgDuaJ8o5lqlVGTQmb6Z2dmDfkTsj4gn0vpx4AVgNbAF2JF22wHcmNa3AF+PzE+B5WkS9euA3RFxKCIOA7uBzbPamglU3KdvZgZMs09f0jrgPcAjwEURsR+yEwOwKu22Gni147C9qWyi8rHvsU1Sn6S+gYGB6VRvQtVy0T/OMjNjGkFf0mLgu8AnIuLYZLuOUxaTlI8uiLg7IjZFxKbe3t6pVm9SlVLBt2EwM2OKQV9SmSzgfzMivpeKD6RuG9LyYCrfC6ztOHwNsG+S8jlXKRWc6ZuZMbXROwLuAV6IiC92PLUTaI/A2Qrc31H+kTSK50rgaOr+eRC4VtKKdAH32lQ256oesmlmBkBpCvtcBfw58Kykp1LZp4DPA/dJugV4BbgpPbcLuAHoB04BHwWIiEOSPgc8lvb7bEQcmpVWnEXWveNM38zsrEE/In7M+P3xANeMs38At07wWtuB7dOp4Gxwpm9mlsnHL3Kd6ZuZAXkJ+s70zcyAnAT9ahq9k/U8mZnlVy6CfqVcJAKGmu7iMbN8y0fQL7Vnz3LQN7N8y0fQL6d5cn0x18xyLh9Bf3jKRF/MNbN8y0XQr7YzfXfvmFnO5SLoO9M3M8vkKug70zezvMtF0B/p3nGmb2b5lougP5zpe/SOmeVcToK+M30zM8hJ0K+W3advZgY5CfrtH2d59I6Z5V0ugn7Vo3fMzICpTZe4XdJBSc91lH1a0q8lPZUeN3Q8d7ukfkk/l3RdR/nmVNYv6bbZb8rEnOmbmWWmkul/Ddg8TvldEbExPXYBSLoMuBl4Vzrmy5KKkorA3wHXA5cBH077zguP3jEzy0xlusQfSVo3xdfbAtwbETXgl5L6gSvSc/0R8RKApHvTvs9Pu8YzUC4WKBbk7h0zy71z6dP/mKRnUvfPilS2Gni1Y5+9qWyi8jNI2iapT1LfwMDAOVRvtGzKRHfvmFm+zTTofwV4O7AR2A/8TSofbwL1mKT8zMKIuyNiU0Rs6u3tnWH1zlRJs2eZmeXZWbt3xhMRB9rrkv4eeCBt7gXWduy6BtiX1icqnxdVz5NrZjazTF/SxR2bfwq0R/bsBG6WVJG0HtgAPAo8BmyQtF5SD9nF3p0zr/b0Zd07zvTNLN/OmulL+hbwfuBCSXuBO4D3S9pI1kXzMvAXABGxR9J9ZBdoG8CtEdFMr/Mx4EGgCGyPiD2z3ppJONM3M5va6J0Pj1N8zyT73wncOU75LmDXtGo3i5zpm5nl5Be5kN10zZm+meVdfoJ+2aN3zMzyE/RLRXfvmFnu5Sfolwvu3jGz3MtN0K+Wir73jpnlXm6CvjN9M7McBX1n+mZmOQr6lXKBQWf6ZpZz+Qn6pQL1ZtBsjXufNzOzXMhN0K+m2bOGPFbfzHIsN0G/PXuW76lvZnmWo6CfZfr+Va6Z5Vlugn61nObJ9cVcM8ux3AT9dqbvWzGYWZ7lJug70zczm0LQTxOfH5T0XEfZSkm7Jb2YlitSuSR9SVJ/mjT9vR3HbE37vyhp69w0Z2LO9M3Mppbpfw3YPKbsNuChiNgAPJS2Aa4nmyJxA7CNbAJ1JK0km3HrfcAVwB3tE8V8qTjTNzM7e9CPiB8Bh8YUbwF2pPUdwI0d5V+PzE+B5Wk+3euA3RFxKCIOA7s580Qyp6rt0TvO9M0sx2bap39RROwHSMtVqXw18GrHfntT2UTlZ5C0TVKfpL6BgYEZVu9M7Uzft2Iwszyb7Qu5GqcsJik/szDi7ojYFBGbent7Z61izvTNzGYe9A+kbhvS8mAq3wus7dhvDbBvkvJ5M9Kn76BvZvk106C/E2iPwNkK3N9R/pE0iudK4Gjq/nkQuFbSinQB99pUNm98GwYzMyidbQdJ3wLeD1woaS/ZKJzPA/dJugV4Bbgp7b4LuAHoB04BHwWIiEOSPgc8lvb7bESMvTg8p9o3XHOmb2Z5dtagHxEfnuCpa8bZN4BbJ3id7cD2adVuFvUUnembmeXmF7mFgugpFpzpm1mu5Sbog+fJNTPLV9AvFX0bBjPLtVwF/aozfTPLuVwF/UrJffpmlm85C/pFah69Y2Y5lqugn3XvONM3s/zKVdDPLuQ60zez/MpX0Hemb2Y5l6ugXy0VfZdNM8u1XAX9Srng++mbWa7lKug70zezvMtV0PdtGMws7/IV9EsF34bBzHItV0G/Wi5SazTJ7gBtZpY/uQr6lVKBVkC96aBvZvl0TkFf0suSnpX0lKS+VLZS0m5JL6blilQuSV+S1C/pGUnvnY0GTEelPTm6+/XNLKdmI9P/g4jYGBGb0vZtwEMRsQF4KG0DXA9sSI9twFdm4b2nperJ0c0s5+aie2cLsCOt7wBu7Cj/emR+CiyXdPEcvP+E2pm+b8VgZnl1rkE/gH+S9LikbansoojYD5CWq1L5auDVjmP3prJRJG2T1Cepb2Bg4ByrN1rFmb6Z5dxZJ0Y/i6siYp+kVcBuST+bZF+NU3bGFdWIuBu4G2DTpk2zesV1uE/fwzbNLKfOKdOPiH1peRD4PnAFcKDdbZOWB9Pue4G1HYevAfady/tPVzvT960YzCyvZhz0JS2StKS9DlwLPAfsBLam3bYC96f1ncBH0iieK4Gj7W6g+VJ1pm9mOXcu3TsXAd+X1H6df4iIf5T0GHCfpFuAV4Cb0v67gBuAfuAU8NFzeO8ZcaZvZnk346AfES8Bl49T/gZwzTjlAdw60/ebDZVSupDrTN/McipXv8itlv3jLDPLt1wFfWf6ZpZ3uQr6zvTNLO9yFfSHM33/OMvMcipnQd+3YTCzfMtV0C8XRUHO9M0sv3IV9CVRKRWd6ZtZbuUq6EN7nlxn+maWT7kL+tVSkcdePszOp/fx+ona+a6Omdm8Ote7bL7p/Nv3ruYbP/0V//FbTwJw6cVL+b13XMBV77iQK9avZGFP7v4kZpYj+k2eJHzTpk3R19c366/bbAXP/vooP+l/nZ/0v07fy4cZarZYUC5yzaWr+OPLL+HfvLN3eFy/mdmbiaTHO2YzHP1cHoP+WKeHmjz28iF2P3+AXc/u542TQyyplLju3W/hjy+/hH/99gsoSgw1WwzWmwzWs2W92aKnVKBaLlItF1lQLlIuinQTOjOz88JBfxoazRb/7xdv8D+f3sc/7nmN44MNigXRbE3t71QQLOwp8b71K/nQ767h6ktXDf8+wMxsPjjoz1Ct0eRH//w6T7xymHKxQLVcoFoqpsy+QE+pwFCjxemO7H+w3uTIqTq7nz/Aa8cGWb6wzJbLL+FDv7uWd69eOuG3gKFGi4PHB3nt6CD7jg7y2tHT7D86yPHBBm9duZC39S7ibRcuZv2Fi1jQ45OImU3MQf88aLaCn/S/znce38uDe16j1mjx2xct4XcuXsLxwQbHB+tpmdZrDcb+UyzqKbK4WuLAsdGjjC5ZVmV97yJ6F1dYsaiHlQt7WJ6WKxaVWdRTolIuUCkVqZQK2aN936F6k1qjlR5NavUWASypllhaLbOkWjrjWkaj2eLI6TqHTw5x6OQQR0/XWdBTZNmC8vBjSbVMsTD33VqtVlCYh/cxezObLOh7qMocKRbE77+zl99/Zy9HT9f5X8/s53tP7OXpV4+wuFpiSaXMW1cuZHEKtssWlLlkeZW3LFvAxcuqvGVZlaXVMpBdc3j5jZO8NHCSlwZO8NLrJ3n5jZM88coRDp8a4vhgY1br3lMqsLRaZkFPgaOn6hyb4usvqZbSdY0CpaKyZSFbFgvZr6ELEgUJpfVSUSxJf48l1RJLF4yceF4/XuO1Y4McODbIgWPZ+usnaiyulHjL0uxvdPGyalpfgASHTw1x9FSdI6fqHDk9xJFTdRqtGHWCWpqWPaUCrx+vcfB4jYHjgxw8XuPgsRqHTw2xclFPx+tn/yYXLavSagXH0gn72GCdY6ezk3at0Uon7SACWhEEUJRY0FNkYU+RhT2ltCyyqFJi5aIeLlxcYeWiHi5YnJ20S8UCrVZw9HSdN9JJ9tDJGodO1jlRq3Oy1uTUUIOTQ01O1bJl+xvmYD07kbe/dZaLBd7Wu4i39y7mHatGHhcs6jnrdadWKzhdb3JyqMHpoSatgFJBFAvqWBYYarY4mv7OR0+3/+51ButNFvYUWVwpZY9qtlxUKTHRObtUyL49l4vZsictAerNFs1W0GgG9Va2Plhvjk6c0nKw0aJU0PBrlIsFyin56axP+3NXLRdoBZwcanCy1uDEYIMTtezRaMVwe4vp81osFBDQaAWtyOrUbAXNCCJi+N95caXEwkqRRWl7Kn/zE0MNjp3O2lKQ+O23LJnS/73pmPdMX9Jm4L8BReCrEfH5ifZ9M2f682mo0eLI6SEOn6xz6OQQp+sNavWObL6RBQGh9A2g41tAmk3s+GD2YTvWEcxODzVYtqCcfZtY1MPyhVlgWragzOl6k6On66Mex9J/9nozaLRaNJrBULNFo9mika6JtCJotVJQDKg1W5zo+NZzesyvpVcsLHNRCvAXLanSu6TCiVqD/UdP89qxGq8dPc3A8Rqdl1x6SgVWLCyzfEEPyxaWKRU0qp5jT5IXLOqhd0mFVUurrFpSYcXCModO1nntWNbFtv/I4Bn1AugpFli6oMSSaplKqYAkBBQKILITW7MVnB5qcmooC9anhprDf4vxLKmWODXUnPQa0sIxJ5EFPcXU7ZgNKqikwQWD9Sa/GDjJLwZOcGpopP5LKiV6Un2LhdEn4sF6a7ieeTGda3bnYuQkpJGTWzH7seix03VODI3+tv+ety7n+//hqhm9129Mpi+pCPwd8IdkE6U/JmlnRDw/n/XoNj2lAquWVFm1pHq+q3LO6s0WJ1LwX7moZ0rDZhvNFgPph3YrFp79mGYrhrPzlYt6KBcn/41iRHBssMGBY4MUC5qwG2yqhhotTtQaHDo5xBsnarxxcih7nKhx5FSdxelbwAWLs5Nt+7G0WmZBuTjt7q1WK9h/bJD+gyfoP3iCVw+dotFq0WxlbWu2glZk65VykUU9RRZWSsPLheUixYJotIJmq5WWWYZbKorlC3tYvqDM8oXZN6jlC3qo9hQ4VWsOZ8ztx8lxujEBguzfsd5sMdRoMdSMbNloIWWBuZyy7GyZ3VJlScrY2/8miyslFvQUabSy49uvV2+2GKy3skw+PY4NtrP6OuViYfibSPvbwKJKiVJRtFox3Ob2oxUx/G2n2P4mkP5dTg9l35BODTU4UUvfyGoNas0W9UaMqlOt2aKSvlkvrWYJRDuRuGhpZUafr7OZ10xf0r8CPh0R16Xt2wEi4r+Mt78zfTOz6Zss05/v2zCsBl7t2N6byoZJ2iapT1LfwMDAvFbOzKzbzXfQH+976aivGhFxd0RsiohNvb2981QtM7N8mO+gvxdY27G9Btg3z3UwM8ut+Q76jwEbJK2X1APcDOyc5zqYmeXWvI7eiYiGpI8BD5IN2dweEXvmsw5mZnk27z/OiohdwK75fl8zM8vhJCpmZnnmoG9mliO/0TdckzQA/OocXuJC4PVZqs6bidudL253vkyl3b8VEeOOef+NDvrnSlLfRL9K62Zud7643flyru12946ZWY446JuZ5Ui3B/27z3cFzhO3O1/c7nw5p3Z3dZ++mZmN1u2ZvpmZdXDQNzPLka4M+pI2S/q5pH5Jt53v+swlSdslHZT0XEfZSkm7Jb2YlivOZx1nm6S1kh6W9IKkPZI+nsq7vd1VSY9Kejq1+zOpfL2kR1K7v51uZth1JBUlPSnpgbSdl3a/LOlZSU9J6ktlM/6sd13Q75iS8XrgMuDDki47v7WaU18DNo8puw14KCI2AA+l7W7SAD4ZEZcCVwK3pn/jbm93Dbg6Ii4HNgKbJV0JfAG4K7X7MHDLeazjXPo48ELHdl7aDfAHEbGxY3z+jD/rXRf0gSuA/oh4KSKGgHuBLee5TnMmIn4EHBpTvAXYkdZ3ADfOa6XmWETsj4gn0vpxskCwmu5vd0TEibRZTo8Arga+k8q7rt0AktYAfwR8NW2LHLR7EjP+rHdj0D/rlIw5cFFE7IcsQAKrznN95oykdcB7gEfIQbtTF8dTwEFgN/AL4EhENNIu3fp5/1vgL4FW2r6AfLQbshP7P0l6XNK2VDbjz/q831p5Hpx1SkbrDpIWA98FPhERx7Lkr7tFRBPYKGk58H3g0vF2m99azS1JHwQORsTjkt7fLh5n165qd4erImKfpFXAbkk/O5cX68ZM31MywgFJFwOk5cHzXJ9ZJ6lMFvC/GRHfS8Vd3+62iDgC/JDsmsZySe0Erhs/71cBfyLpZbLu2qvJMv9ubzcAEbEvLQ+Sneiv4Bw+690Y9D0lY9berWl9K3D/eazLrEv9ufcAL0TEFzue6vZ296YMH0kLgA+QXc94GPhQ2q3r2h0Rt0fEmohYR/b/+QcR8Wd0ebsBJC2StKS9DlwLPMc5fNa78he5km4gywTaUzLeeZ6rNGckfQt4P9ntVg8AdwD/A7gPeCvwCnBTRIy92PumJen3gP8LPMtIH++nyPr1u7nd/4Lsol2RLGG7LyI+K+ltZBnwSuBJ4N9FRO381XTupO6d/xQRH8xDu1Mbv582S8A/RMSdki5ghp/1rgz6ZmY2vm7s3jEzswk46JuZ5YiDvplZjjjom5nliIO+mVmOOOibmeWIg76ZWY78f4j2dFri7ftnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
